#include <iostream>
#include <stdio.h>
#include <curl/curl.h>
#include <curl/types.h>
#include <curl/easy.h>

#include "URL.h"

#include "DownloadParser.h"
#include "../classes_dados/Repository.h"
#include "../classes_logger/DebugLogger.h"
#include "../classes_dados/Hasher.h"

using namespace std;

int DownloadParser::numSitesUrl = 0;
bool DownloadParser::continueParsing = true;
boost::mutex DownloadParser::continueParsingMutex;



string DownloadParser::getRobot(){                 
	try{
		//Teste de Tratamento	
		if( (site_url.size() > 300) || (site_url.size() <= 0) ) return ""; 
		string site_robot = site_url + "robots.txt";
		
		curl = curl_easy_init();
		if(curl){
			curl_easy_setopt(curl, CURLOPT_URL, site_robot.c_str());
			curl_config();	
			curl_easy_cleanup(curl);
		}
		
		// Tratamento de erro simplista
		if(res_code == 200) return data_site;
		else                return "";
	}catch(...){
		cout << "Vixi" << endl;
		return " ";
	}
}

string DownloadParser::getPage(){
	try{	
		//Teste de Tratamento	
		if( (site_url.size() > 300) || (site_url.size() <= 0) ) return ""; 
		
		curl = curl_easy_init();
		if(curl){
			curl_easy_setopt(curl, CURLOPT_URL, site_url.c_str());
			curl_config();
			curl_easy_cleanup(curl);
		}
		
		if(res_code == 301 || res_code == 302){
			uint index = header_site.find("Location:");
			if(index != string::npos){
				uint index2;
				for(uint i=index; i<header_site.size(); i++) if(header_site[i] == (char) 13 || header_site[i] == (char) 10){ index2=i; break;}
				data_site = "<html>\n<meta name=\"robots\" content=\"noindex\"> \n<body>\n    <a  href=\"" 
					        + header_site.substr(index+10, index2-(index+10)) + 
					        "\"> Header </a>\n</body>\n</html>";
				return data_site;
			}
		}

		// Tratamento de erro simplista
		if(res_code == 200) return data_site;
		else                return "";
	}catch(...){
		cout << "Vixi" << endl;
		return "";
	}	
	
}



size_t DownloadParser::WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *userp){
	size_t realsize = size * nmemb;
	try{
		
		if( (ptr == NULL) || (nmemb==0) )  return realsize;
		string *data_str = (string *) userp;
		data_str->append((char *)ptr, nmemb); 
		if (data_str->size() > 300*1024) return -1;
		
	}catch(...){
		cout << "Vixi Especial..." << endl;
	} 
	return realsize;
}

size_t DownloadParser::headerCallback(void *ptr, size_t size, size_t nmemb, void *userp){
	size_t realsize = size * nmemb;
	try{

		if( (ptr == NULL) || (nmemb==0) )  return realsize;
		string *data_str = (string *) userp;
		data_str->append((char *)ptr, nmemb); 
		if (data_str->size() > 300*1024) return -1;
		
	}catch(...){
		cout << "Vixi Especial..." << endl;
	}
	return realsize;
}





void DownloadParser::trim(string &str) {
    try{
//    cout << "antes trim: " << str << endl;
    string forbidden = " \t\r";
    while (str.find_last_of(forbidden) == str.size()-1 && str.size() > 0) {
        str.erase(str.size()-1);
//        cout << "inicio trim:  " << str << endl;
    }
    while (str.find_first_of(forbidden) == 0 && str.size() > 0) {
        str.erase(0,1);
//        cout << "meio trim:  " << str << endl;
    }
//    cout << "fim trim: " << str << endl;
    }catch(...){
        cout << "Baggio de novo, agora no Robots ... " << endl;
    }
}

bool DownloadParser::checkIgnoreCase(string str, string compared) {
    trim(str);
    if (str.size() < compared.size()) return false;
    
    for (uint i = 0; i < compared.size(); i++) {
        if ( !(tolower(str[i]) == compared[i]) )
            return false;
    }
    return true;
}



bool DownloadParser::isFollowPage(string str) {
	

	long long faltanteD0 = urlFileDealer->faltanteNoDiscoZero();
	long long faltanteTotal = (long long)(10000000) - Repository::getContador();

	
	{
		boost::mutex::scoped_lock continueParsingLock (continueParsingMutex);
		bool aux = continueParsing;
		
	    if (continueParsing && (faltanteD0 > faltanteTotal) && (faltanteD0 > 100000) )
	    	continueParsing = false;
	    else if ( !continueParsing && (faltanteD0 < 50000) && (faltanteTotal > 0) )
	    	continueParsing = true;
	    
	    if (aux != continueParsing) {
	    	if (continueParsing) {
	    		cout << "\n\nEstou a fazer parsing, Manuel" << endl;
	    		cout << "-> faltante: " << faltanteD0 << endl;
	    		cout << "-> faltanteTotal: " << faltanteTotal << endl;
	    	}
	    	else {
	    		cout << "\n\nNao estou mais a fazer parsing, oras pois" << endl;
	    		cout << ">> faltante: " << faltanteD0 << endl;
	    		cout << ">> faltanteTotal: " << faltanteTotal << endl;
	    	}
	    }
	}
	
	uint ltLocation;
    bool strFound = false;
    uint headBeginIndex;
    uint gtLocation = 0;
    string temp;
    
    //acha <head>
    do {
        ltLocation= str.find_first_of('<', gtLocation);
        if (ltLocation != string::npos)
        {
            gtLocation = str.find_first_of('>', ltLocation);
            if (gtLocation != string::npos) {
                temp = str.substr(ltLocation + 1, gtLocation -ltLocation -1);
                strFound = checkIgnoreCase(temp, "head");
            }
        }
        else
            return true;
    }while(!strFound);
    
    headBeginIndex = gtLocation;
    
    //acha </head>
    do {
        ltLocation= str.find("</", gtLocation);
        if (ltLocation != string::npos)
        {
            gtLocation = str.find_first_of('>', ltLocation);
            if (gtLocation != string::npos) {
                temp = str.substr(ltLocation + 1, gtLocation - ltLocation - 1);
                strFound = checkIgnoreCase(temp, "/head");
            }
        }
        else
            return true;
    }while(!strFound);
    
    str = str.substr(headBeginIndex, gtLocation - headBeginIndex);    
    for (uint i = 0; i < str.size(); i++)
        str[i] = tolower(str[i]);    
    
    gtLocation = 0;
    //acha <meta ... nofollow>
    do {
        ltLocation= str.find_first_of('<', gtLocation);
        if (ltLocation != string::npos)
        {
            gtLocation = str.find_first_of('>', ltLocation);
            if (gtLocation != string::npos) {
                temp = str.substr(ltLocation + 1, gtLocation - ltLocation - 1);
                strFound = ( (temp.find("meta") != string::npos) && (temp.find("nofollow") != string::npos) );
            }
        }
        else
            return true;
    }while(!strFound);
    
    return false;
}




list<URL>  DownloadParser::parser(string buffer_page){
	
	unsigned int count_link = 0;	
	list<string> buffer_links;
	list<URL> buffer_urls;
    bool followPage = isFollowPage(buffer_page);
    
    if (!followPage || !continueParsing) {
        if (!followPage)
        	cout << "Snif, nao posso seguir links de " << pagina_at << endl;
        return buffer_urls;
    }	
	
	std::string::size_type pos_final;
	std::string::size_type pos_atual;
	
	//acha posicao final do arquivo	url_rejeita = "()'!";
	pos_final = buffer_page.size();
	pos_atual = buffer_page.find(s_href, 0);
	do{
		std::string::size_type pos_aspa_begin;
		std::string::size_type pos_aspa_end;
		if(pos_atual >= pos_final) continue;
		//pega o link, se existe
		pos_aspa_begin = buffer_page.find("\"", pos_atual);
		pos_aspa_end = buffer_page.find("\"",pos_aspa_begin+1);
		buffer_links.push_back(buffer_page.substr(pos_aspa_begin+1, pos_aspa_end-pos_aspa_begin-1));
		pos_atual = buffer_page.find(s_href, pos_atual+1);
		count_link++;
	}while(pos_atual < pos_final);
	
	// Aplica regras para extrair falsos links
	rejeita_links(buffer_links);
	// Normalize
	buffer_urls = eliminaManjados( normalisa.normalize(buffer_links) );  // canoniza URLs
	
	uint tamanhoBuffer = buffer_urls.size();
	URL temp;
	for(uint i=0; i<tamanhoBuffer; i++){
		temp = buffer_urls.front();
		buffer_urls.pop_front();
		if(temp.toString().size() <= 300){
			buffer_urls.push_back(temp);
		}
	}
	return buffer_urls;
}



list<URL> DownloadParser::getNewURLs(void){
	list<URL> vazia;
	
	buffer_links.clear();
	buffer_page.clear();
	
	//carrega site
	buffer_page = getPage();
	
	//se vazio, n�o cria arquivo e continua
	if(buffer_page.size() == 0) return vazia; 
	
	//grava site
	disco->store(site_url, buffer_page);

	pagina_at = site_url;
	list<URL> rec_list = parser(buffer_page);

	//Aqui grava info para fazer backup
//	unsigned long long _hash = Hasher::hash64FromString(site_url);
//	disco->writeHashFile(_hash);

	return rec_list;
}



list<URL> DownloadParser::eliminaManjados(list<URL> linkList) {
	list<string> listaDeFormatosManjados;
	list<URL> listaFiltrada;
	try{	
		
		
		//Multimidia
		listaDeFormatosManjados.push_front(".jpg");
		listaDeFormatosManjados.push_front(".gif");
		listaDeFormatosManjados.push_front(".bmp");
		listaDeFormatosManjados.push_front(".swf");
		listaDeFormatosManjados.push_front(".mp3");
		listaDeFormatosManjados.push_front(".avi");
		listaDeFormatosManjados.push_front(".ico");
		listaDeFormatosManjados.push_front(".mpeg");
		listaDeFormatosManjados.push_front(".mpg");
		listaDeFormatosManjados.push_front(".jpeg");
		listaDeFormatosManjados.push_front(".wav");
		
		//Zips
		listaDeFormatosManjados.push_front(".rpm");
		listaDeFormatosManjados.push_front(".tar");
		listaDeFormatosManjados.push_front(".gz");
		listaDeFormatosManjados.push_front(".zip");
		listaDeFormatosManjados.push_front(".rar");
		
		//Outros
		listaDeFormatosManjados.push_front(".pdf");
		listaDeFormatosManjados.push_front(".doc");
		listaDeFormatosManjados.push_front(".pps");
		listaDeFormatosManjados.push_front(".png");
		listaDeFormatosManjados.push_front(".cdr");
		listaDeFormatosManjados.push_front(".css");
		
		//Sistema
		listaDeFormatosManjados.push_front(".exe");
		listaDeFormatosManjados.push_front(".bat");
		listaDeFormatosManjados.push_front(".o");
		
	
		list<string>::iterator stringIt;
		list<URL>::iterator urlIt;
		
		string path;
		for (urlIt=linkList.begin(); urlIt!=linkList.end(); urlIt++) {
			listaFiltrada.push_front(*urlIt);
			for (stringIt=listaDeFormatosManjados.begin(); stringIt!=listaDeFormatosManjados.end(); stringIt++) {
				path = urlIt->get_path();
				if (path.size() < stringIt->size()) 
					continue;
				if ( path.rfind(*stringIt) != string::npos) {
					listaFiltrada.pop_front();
					break;
				}
			}
		}
		
	}catch(...){
		cout << "Vixi..." << endl;
	}
	return listaFiltrada;
}


string  DownloadParser::achaDominio (string str) {
	try{
		uint index = str.find("://");
		if (index != string::npos) { 
			str = str.substr( index+3, str.size() );
		}
		uint slashIndex = str.find("/");
		if (slashIndex != string::npos) 
			return str.substr(0, slashIndex);
	}catch(...){
		cout << "Vixi..." << endl;
	}
	return str;
}



void DownloadParser::rejeita_links(std::list <std::string> &lista_in){
	std::string temp_url;
	string url_rejeita;
	int buffer_links_size = (int) lista_in.size();
	try{		
		
		url_rejeita = "()'!@{}|\\^[]`<>";
		//referencia para links relativos
		uint pos_barra = pagina_at.find_last_of("/"); 
		if( pos_barra != std::string::npos) pagina_at.erase(pos_barra);
		//Grava em arquivo a lista
		for( int i=0; i<buffer_links_size; i++){
			temp_url = lista_in.front();
			if( temp_url.find_first_of(url_rejeita) == std::string::npos
				&& temp_url.find_first_of("./") != std::string::npos ){
					// Tira carecteres indesejados
					for(unsigned int t=0; t < temp_url.size(); t++)
						if(temp_url.at(t) == ' ' || temp_url.at(t) == '\n'){
							temp_url.erase(t,1); // tira caractere
							t--;           // volta uma pos.
						}
					//resolve links relativos
					if (temp_url.at(0) == '/') {
						temp_url = achaDominio(pagina_at) + temp_url;
					}
					
					if(temp_url.at(0) == '.'){
						for(uint t=0; t<=temp_url.size(); t++)
							if(temp_url.at(0) == '/' || temp_url.at(0) == '.'){
								if(temp_url.at(0) == '.' && temp_url.at(1) == '.'){
									temp_url.erase(0,2);
									pos_barra = pagina_at.find_last_of("/"); 
									if( pos_barra != std::string::npos){
										if( pagina_at.find_last_of("/") > pagina_at.find_last_of(".")  
											&& pagina_at.find_last_of("/") != std::string::npos
											&& pagina_at.find_last_of("/") != std::string::npos) pagina_at.erase(pos_barra); //retorna dir se puder
									}
								}
								temp_url.erase(0,1);
								if(temp_url.find_first_of("./") == std::string::npos){
									temp_url = pagina_at + '/' + temp_url; 
									break;
								}
							}else{
								temp_url = pagina_at + '/' + temp_url;
								break;
							}
					}else if(   temp_url.find("://") == std::string::npos &&
							  ( temp_url.find_first_of("/") < temp_url.find_first_of(".") ||
							    temp_url.find_first_of("/") == std::string::npos )){
						temp_url = pagina_at + '/' + temp_url;
					}
					lista_in.push_back(temp_url);
					lista_in.pop_front();
			}
			
			else{
				lista_in.pop_front();
			}
		}
		
	}catch(...){ // Gambi;
	}
}




int DownloadParser::curl_config(void) {
	 CURLcode res;
	 struct data config;
	 config.trace_ascii = 1; /* enable ascii tracing */
	 cleanBuffer();
	 if(curl) {
		 curl_easy_setopt(curl, CURLOPT_VERBOSE, 0);
	     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
	     curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data_site);
	     curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback );
	     curl_easy_setopt(curl, CURLOPT_HEADERDATA,&header_site);
	   
	   //Tamanho maximo para arquivo (em bytes)
	   curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, 300*1024); //limita 300 kbytes
	   
	   //Redirecionamentos
	   curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
	   
	   curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5);
	   
	   //deve setar para TRUE se em multi-Thread
	   curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);	   
	   
	   res = curl_easy_perform(curl);
	   res = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &res_code);    
	 }
	 return 0;
}

